{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read Data Sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-01T17:50:17.211667Z",
     "start_time": "2017-06-01T17:50:16.779111Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "from collections import namedtuple\n",
    "pd.set_option(\"display.max_rows\",35)\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-01T17:50:17.317181Z",
     "start_time": "2017-06-01T17:50:17.213260Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class dataset:\n",
    "    kdd_train_2labels = pd.read_pickle(\"dataset/kdd_train_2labels.pkl\")\n",
    "    kdd_test_2labels = pd.read_pickle(\"dataset/kdd_test_2labels.pkl\")\n",
    "    \n",
    "    kdd_train_5labels = pd.read_pickle(\"dataset/kdd_train_5labels.pkl\")\n",
    "    kdd_test_5labels = pd.read_pickle(\"dataset/kdd_test_5labels.pkl\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-01T17:50:17.324136Z",
     "start_time": "2017-06-01T17:50:17.318752Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(125973, 124)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.kdd_train_2labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-01T17:50:17.331210Z",
     "start_time": "2017-06-01T17:50:17.325910Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22544, 124)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.kdd_test_2labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-01T17:50:18.165473Z",
     "start_time": "2017-06-01T17:50:17.333217Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(125973, 122)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import model_selection as ms\n",
    "from sklearn import preprocessing as pp\n",
    "\n",
    "class preprocess:\n",
    "    \n",
    "    output_columns_2labels = ['is_Attack','is_Normal']\n",
    "    \n",
    "    x_input = dataset.kdd_train_2labels.drop(output_columns_2labels, axis = 1)\n",
    "    y_output = dataset.kdd_train_2labels.loc[:,output_columns_2labels]\n",
    "\n",
    "    x_test_input = dataset.kdd_test_2labels.drop(output_columns_2labels, axis = 1)\n",
    "    y_test_output = dataset.kdd_test_2labels.loc[:,output_columns_2labels]\n",
    "    #y_test_output = 1 - dataset.kdd_test_2labels_y \n",
    "    \n",
    "    ss = pp.StandardScaler()\n",
    "\n",
    "    x_train = ss.fit_transform(x_input)\n",
    "    x_test = ss.transform(x_test_input)\n",
    "\n",
    "    y_train = y_output.values\n",
    "    y_test = y_test_output.values\n",
    "\n",
    "    x_train_all = x_train #[y_output.is_Normal == 1]\n",
    "    y_train_all = 1 - np.argmax(y_output.values, axis = 1) # Inverting because here H0 = Normal and H1 is Attack. \n",
    "    #y_output.is_Normal.values #[y_output.is_Normal == 1]\n",
    "    \n",
    "    x_test_all = x_test\n",
    "    y_test_all = 1- np.argmax(y_test_output.values, axis = 1) # Inverting because here H0 = Normal and H1 is Attack.\n",
    "    \n",
    "preprocess.x_train_all.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Data Reduction by:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-01T17:50:19.265071Z",
     "start_time": "2017-06-01T17:50:18.167051Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-01T17:50:19.270935Z",
     "start_time": "2017-06-01T17:50:19.266679Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class network(object):\n",
    "    \n",
    "    input_dim = 122\n",
    "    classes = 1\n",
    "    hidden_dim = 122\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-01T17:50:19.549397Z",
     "start_time": "2017-06-01T17:50:19.272643Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class discriminator(network):\n",
    "    \n",
    "    def __init__(self, hidden_layers, f):\n",
    "        \n",
    "\n",
    "        tf.reset_default_graph()\n",
    "        input_dim = self.input_dim\n",
    "        classes = self.classes\n",
    "        hidden_dim = self.hidden_dim #f\n",
    "\n",
    "        self.x_real = tf.placeholder(\"float\", shape=[None, input_dim]) # Only normal type Data\n",
    "        self.y_real_ = tf.placeholder(\"float\", shape=[None, classes]) # Output for H0 hypothes\n",
    "        #self.y_ = tf.placeholder(\"float\", shape=[None, classes])\n",
    "\n",
    "        self.x_random = tf.placeholder(\"float\", shape=[None, input_dim]) # Data for H1 hypothesis\n",
    "        self.y_fake_ = tf.placeholder(\"float\", shape=[None, classes]) # Output for H1 hypothesis\n",
    "\n",
    "        self.keep_prob = tf.placeholder(\"float\")\n",
    "        self.learning_rate = tf.placeholder(\"float\")\n",
    "\n",
    "        def discriminator_network(x, reuse=False):\n",
    "            with tf.variable_scope(\"discriminator\", reuse=reuse):\n",
    "                hidden = tf.layers.dense(x, hidden_dim, activation = tf.nn.relu, kernel_regularizer=tf.nn.l2_loss)\n",
    "                hidden = tf.nn.dropout(hidden, self.keep_prob)\n",
    "\n",
    "                for h in range(hidden_layers - 1):\n",
    "                    hidden = tf.layers.dense(hidden, hidden_dim, activation = tf.nn.relu, kernel_regularizer=tf.nn.l2_loss)\n",
    "                    hidden = tf.nn.dropout(hidden, self.keep_prob)\n",
    "\n",
    "            y = tf.layers.dense(hidden, classes, activation=tf.nn.sigmoid)\n",
    "            return y\n",
    "        \n",
    "        def generator_network(x, reuse=False):\n",
    "            with tf.variable_scope('generator', reuse=reuse):\n",
    "                hidden = tf.layers.dense(x, hidden_dim, activation = tf.nn.relu, kernel_regularizer=tf.nn.l2_loss)\n",
    "                hidden = tf.nn.dropout(hidden, self.keep_prob)\n",
    "\n",
    "                for h in range(hidden_layers - 2):\n",
    "                    hidden = tf.layers.dense(hidden, hidden_dim, activation = tf.nn.relu, kernel_regularizer=tf.nn.l2_loss)\n",
    "                    hidden = tf.nn.dropout(hidden, self.keep_prob)\n",
    "\n",
    "            y = tf.layers.dense(hidden, input_dim, activation=tf.nn.relu)\n",
    "            return y\n",
    "\n",
    "        x_fake = generator_network(self.x_random)\n",
    "\n",
    "        self.y_real = discriminator_network(self.x_real)\n",
    "        y_fake = discriminator_network(x_fake, reuse=True)\n",
    "\n",
    "        \n",
    "        #loss_real = tf.losses.sigmoid_cross_entropy(self.y_real_, self.y)\n",
    "        #loss_fake = tf.losses.sigmoid_cross_entropy(self.y_fake_, y_fake)\n",
    "\n",
    "        loss_real = tf.losses.mean_squared_error(self.y_real_, self.y_real)\n",
    "        loss_fake = tf.losses.mean_squared_error(self.y_fake_, y_fake)\n",
    "        \n",
    "        self.loss = loss_real + loss_fake\n",
    "\n",
    "        #correct_prediction = tf.equal(self.y_, self.y)\n",
    "        #self.tf_accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name = \"Accuracy\")\n",
    "\n",
    "        d_optimizer = tf.train.AdamOptimizer(self.learning_rate)\n",
    "        g_optimizer = tf.train.AdamOptimizer(self.learning_rate)\n",
    "\n",
    "        #gradients, variables = zip(*optimizer.compute_gradients(self.loss))\n",
    "        #gradients = [\n",
    "        #    None if gradient is None else tf.clip_by_value(gradient, -1, 1)\n",
    "        #    for gradient in gradients]\n",
    "        #self.train_op = optimizer.apply_gradients(zip(gradients, variables))\n",
    "        self.d_train_op = d_optimizer.minimize(self.loss)\n",
    "        self.g_train_op = g_optimizer.minimize(loss_fake)\n",
    "        self.loss_f = loss_fake\n",
    "        # add op for merging summary\n",
    "        #self.summary_op = tf.summary.merge_all()\n",
    "        #self.pred = tf.argmax(self.y, axis = 1)\n",
    "        #self.actual = tf.argmax(self.y_real_, axis = 1)\n",
    "\n",
    "        # add Saver ops\n",
    "        self.saver = tf.train.Saver()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-01T17:50:20.203304Z",
     "start_time": "2017-06-01T17:50:19.551362Z"
    },
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import collections\n",
    "\n",
    "class Train:    \n",
    "    alpha = 0.8\n",
    "    result = namedtuple(\"score\", ['epoch', 'no_of_features','hidden_layers','train_score', 'test_score'])\n",
    "\n",
    "    predictions = {}\n",
    "\n",
    "    results = []\n",
    "    best_acc = 0\n",
    "    \n",
    "    def train(epochs, h, f, lrs):\n",
    "        batch_iterations = 400\n",
    "        train_loss = None\n",
    "        train_loss_g = None\n",
    "        Train.best_acc = 0\n",
    "        \n",
    "        d = discriminator(h, f)\n",
    "        #g = generator(h)\n",
    "        \n",
    "        os.makedirs(\"dataset/tf_GAN_with_Dense_nsl_kdd/hidden_layers_{}_features_count_{}\".format(epochs,h,f),\n",
    "                    exist_ok = True)\n",
    "        \n",
    "        d.sess = tf.Session()\n",
    "        \n",
    "        #summary_writer_train = tf.summary.FileWriter('./logs/kdd/VAE/training', graph=sess.graph)\n",
    "        #summary_writer_valid = tf.summary.FileWriter('./logs/kdd/VAE/validation')\n",
    "\n",
    "        d.sess.run(tf.global_variables_initializer())\n",
    "\n",
    "        for lr in lrs:\n",
    "            for epoch in range(1, (epochs+1)):\n",
    "                x_train, x_valid, y_train, y_valid, = ms.train_test_split(preprocess.x_train_all, \n",
    "                                                                          preprocess.y_train_all, \n",
    "                                                                          test_size=0.1)\n",
    "                \n",
    "                x_train = x_train[y_train == 0]\n",
    "                y_train = y_train[y_train == 0]\n",
    "                \n",
    "                y_train = np.reshape(y_train, (-1,1))\n",
    "                y_train_normal = np.zeros_like(y_train)\n",
    "                y_train_attack = np.ones_like(y_train)\n",
    "                x_train_attack_rnd = np.random.normal(size=x_train.shape)\n",
    "\n",
    "                y_valid = np.reshape(y_valid, (-1,1))\n",
    "                y_valid_normal = np.zeros_like(y_valid)\n",
    "                y_valid_attack = np.ones_like(y_valid)\n",
    "                x_valid_attack_rnd = np.random.normal(size=x_valid.shape)\n",
    "\n",
    "                y_test_all = np.reshape(preprocess.y_test_all, (-1,1))\n",
    "                y_test_normal = np.zeros_like(y_test_all)\n",
    "                y_test_attack = np.ones_like(y_test_all)\n",
    "                x_test_attack_rnd = np.random.normal(size=preprocess.x_test_all.shape)\n",
    "\n",
    "                batch_indices = np.array_split(np.arange(x_train.shape[0]), batch_iterations)\n",
    "\n",
    "                for i in batch_indices:\n",
    "\n",
    "                    def train_batch():\n",
    "                        nonlocal train_loss\n",
    "                        nonlocal train_loss_g\n",
    "\n",
    "                        # Passing Normal (real) and Attack (fake) Traffic and training together\n",
    "                        d_loss, _, _ = d.sess.run([d.loss, d.d_train_op, d.g_train_op], \n",
    "                                              feed_dict={d.x_real: x_train[i,:],\n",
    "                                                         #d.y_:y_train[i,:],\n",
    "                                                         d.y_real_: y_train_normal[i,:],\n",
    "                                                         d.x_random: x_train_attack_rnd[i,:],\n",
    "                                                         d.y_fake_: y_train_attack[i,:],\n",
    "                                                         d.keep_prob:1,\n",
    "                                                         d.learning_rate:lr})\n",
    "                        g_loss, _ = d.sess.run([d.loss_f, d.g_train_op], \n",
    "                                              feed_dict={d.x_real: x_train[i,:],\n",
    "                                                         d.y_:y_train[i,:],\n",
    "                                                         d.y_real_: y_train_normal[i,:],\n",
    "                                                         d.x_random: x_train_attack_rnd[i,:],\n",
    "                                                         d.y_fake_: y_train_attack[i,:],\n",
    "                                                         d.keep_prob:1,\n",
    "                                                         d.learning_rate:lr})\n",
    "\n",
    "                        #Train Generator\n",
    "                        #g.sess.run([g.train_op], feed_dict={g.loss:d_loss_attack_fake})\n",
    "\n",
    "                        train_loss = d_loss\n",
    "                        train_loss_g = g_loss\n",
    "\n",
    "                    train_batch()\n",
    "                    #summary_writer_train.add_summary(summary_str, epoch)\n",
    "                    while((train_loss > 1e4 or np.isnan(train_loss)) and epoch > 1):\n",
    "                        print(\"Step {} | Training Loss: d-{:.6f};g-{}\".format(epoch, train_loss, train_loss_g))\n",
    "                        d.saver.restore(sess, \n",
    "                                          tf.train.latest_checkpoint('dataset/tf_GAN_with_Dense_nsl_kdd/hidden_layers_{}_features_count_{}'\n",
    "                                                                     .format(epochs,h,f)))\n",
    "                        train_batch()\n",
    "\n",
    "\n",
    "                y_valid_pred = d.sess.run(d.y, #net.summary_op \n",
    "                                                      feed_dict={d.x_real: x_valid, \n",
    "                                                                 d.y_real_: y_valid_normal,\n",
    "                                                                 d.y_:y_valid,\n",
    "                                                                 d.x_random: x_valid_attack_rnd,\n",
    "                                                                 d.y_fake_: y_valid_attack,\n",
    "                                                                 d.keep_prob:1,\n",
    "                                                                 d.learning_rate:lr})\n",
    "                #summary_writer_valid.add_summary(summary_str, epoch)\n",
    "\n",
    "\n",
    "                y_test_pred = d.sess.run(d.y, \n",
    "                                                feed_dict={d.x_real: preprocess.x_test_all, \n",
    "                                                         d.y_real_: y_test_normal,\n",
    "                                                         d.y_: y_test_all,\n",
    "                                                         d.x_random: x_test_attack_rnd,\n",
    "                                                         d.y_fake_: y_test_attack,\n",
    "                                                         d.keep_prob:1,\n",
    "                                                         d.learning_rate:lr})\n",
    "\n",
    "                def get_accuracy(y_pred, y_actual):\n",
    "                    y_attack = np.logical_or(y_pred > Train.alpha, y_pred < -Train.alpha)\n",
    "                    y_normal = np.logical_or(y_pred < Train.alpha, y_pred > -Train.alpha)\n",
    "                    \n",
    "                    pred_value = np.argmax([y_normal, y_attack], axis = 0)\n",
    "                    accuracy = np.mean(np.equal(y_actual, pred_value))\n",
    "                    \n",
    "                    pred_value = np.squeeze(pred_value)\n",
    "                    y_attack = np.squeeze(y_attack)\n",
    "                    y_normal = np.squeeze(y_normal)\n",
    "                    \n",
    "                    return accuracy, pred_value, y_attack, y_normal\n",
    "                    \n",
    "                valid_accuracy, _,_,_ = get_accuracy(y_valid_pred, y_valid)\n",
    "                accuracy, pred_value, y_attack, y_normal = get_accuracy(y_test_pred, y_test_all)\n",
    "                \n",
    "                print(\"Step {} | Training Loss: d-{:.6f}, g-{:.6f} | Validation Accuracy: {:.6f} | Test Accuracy: {:.6f}\".format(epoch, train_loss, train_loss_g, valid_accuracy, accuracy))\n",
    "                \n",
    "                if accuracy > Train.best_acc:\n",
    "                    \n",
    "                    actual_value = y_test_normal\n",
    "                    \n",
    "                    Train.best_acc = accuracy\n",
    "                    Train.pred_value = pred_value\n",
    "                    Train.actual_value = actual_value\n",
    "                    Train.best_parameters = \"Hidden Layers:{}, Features Count:{}\".format(h, f)\n",
    "                                   \n",
    "                    if not (np.isnan(train_loss)):\n",
    "                        d.saver.save(d.sess, \n",
    "                                   \"dataset/tf_GAN_with_Dense_nsl_kdd/hidden_layers_{}_features_count_{}\".format(h,f),\n",
    "                                    global_step = epochs)\n",
    "                    curr_pred = pd.DataFrame({\"Attack_prob\":y_attack, \"Normal_prob\":y_normal, \"Prediction\":pred_value})\n",
    "                    Train.predictions.update({\"{}_{}_{}\".format(epochs*len(lrs),f,h):(curr_pred, \n",
    "                                               Train.result(epochs*len(lrs), f, h,valid_accuracy, accuracy))})\n",
    "\n",
    "                    #Train.results.append(Train.result(epochs, f, h,valid_accuracy, accuracy))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-01T17:50:40.954591Z",
     "start_time": "2017-06-01T17:50:20.205352Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Layer Attributes - epochs:5 hidden layers:2 features count:4\n",
      "Step 1 | Training Loss: d-0.000000, g-0.000000 | Validation Accuracy: 0.535720 | Test Accuracy: 0.430758\n",
      "Step 2 | Training Loss: d-0.000000, g-0.000000 | Validation Accuracy: 0.534450 | Test Accuracy: 0.430758\n",
      "Step 3 | Training Loss: d-0.000000, g-0.000000 | Validation Accuracy: 0.531513 | Test Accuracy: 0.430758\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-fae4d8d2c53f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mitertools\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mclass\u001b[0m \u001b[0mHyperparameters\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;31m#    features_arr = [2, 4, 8, 16, 32, 64, 128, 256]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m#    hidden_layers_arr = [2, 4, 6, 10]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mfeatures_arr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m8\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m16\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m32\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-10-fae4d8d2c53f>\u001b[0m in \u001b[0;36mHyperparameters\u001b[0;34m()\u001b[0m\n\u001b[1;32m     11\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Current Layer Attributes - epochs:{} hidden layers:{} features count:{}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mh\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m         \u001b[0mTrain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m1e-2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1e-4\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-9-2e57a4fcfd47>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(epochs, h, f, lrs)\u001b[0m\n\u001b[1;32m     46\u001b[0m                 \u001b[0my_valid_normal\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_valid\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m                 \u001b[0my_valid_attack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mones_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_valid\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 48\u001b[0;31m                 \u001b[0mx_valid_attack_rnd\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnormal\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mx_valid\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     49\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m                 \u001b[0my_test_all\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpreprocess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0my_test_all\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import itertools\n",
    "class Hyperparameters:\n",
    "#    features_arr = [2, 4, 8, 16, 32, 64, 128, 256]\n",
    "#    hidden_layers_arr = [2, 4, 6, 10]\n",
    "    features_arr = [4, 8, 16, 32]\n",
    "    hidden_layers_arr = [2, 4, 6]\n",
    "\n",
    "    epochs = [5]\n",
    "    \n",
    "    for e, h, f in itertools.product(epochs, hidden_layers_arr, features_arr):\n",
    "        print(\"Current Layer Attributes - epochs:{} hidden layers:{} features count:{}\".format(e,h,f))\n",
    "        \n",
    "        Train.train(e, h, f, [1e-2, 1e-4])\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-01T17:50:40.955146Z",
     "start_time": "2017-06-01T17:50:14.751Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dict1 = {}\n",
    "dict2 = []\n",
    "for k, (v1, v2) in Train.predictions.items():\n",
    "    dict1.update({k: v1})\n",
    "    dict2.append(v2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-01T17:50:40.955857Z",
     "start_time": "2017-06-01T17:50:14.756Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Train.predictions = dict1\n",
    "Train.results = dict2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-01T17:50:40.956460Z",
     "start_time": "2017-06-01T17:50:14.761Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_results = pd.DataFrame(Train.results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-01T17:50:40.957015Z",
     "start_time": "2017-06-01T17:50:14.766Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_results.sort_values(by = 'test_score', ascending = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-01T17:50:40.957562Z",
     "start_time": "2017-06-01T17:50:14.770Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.Panel(Train.predictions).to_pickle(\"dataset/tf_GAN_with_Dense_nsl_kdd_predictions.pkl\")\n",
    "df_results.to_pickle(\"dataset/tf_GAN_with_Dense_nsl_kdd_scores.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-01T17:50:40.958510Z",
     "start_time": "2017-06-01T17:50:14.773Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import itertools\n",
    "\n",
    "def plot_confusion_matrix(cm, classes,\n",
    "                          normalize=False,\n",
    "                          title='Confusion matrix',\n",
    "                          cmap=plt.cm.Blues):\n",
    "    \"\"\"\n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "    np.set_printoptions(precision=4)\n",
    "\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        print(\"Normalized confusion matrix\")\n",
    "    else:\n",
    "        print('Confusion matrix, without normalization')\n",
    "\n",
    "    print(cm)\n",
    "\n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, cm[i, j].round(4),\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "\n",
    "def plot(actual_value, pred_value):\n",
    "    from sklearn.metrics import confusion_matrix\n",
    "    cm_2labels = confusion_matrix(y_pred = pred_value, y_true = actual_value)\n",
    "    plt.figure(figsize=[6,6])\n",
    "    plot_confusion_matrix(cm_2labels, preprocess.output_columns_2labels, normalize = True,\n",
    "                         title = Train.best_parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-01T17:50:40.959432Z",
     "start_time": "2017-06-01T17:50:14.776Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plot(actual_value = Train.actual_value, pred_value = Train.pred_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "_draft": {
   "nbviewer_url": "https://gist.github.com/7d1ace18a82178e15ece8fc5252fce88"
  },
  "anaconda-cloud": {},
  "gist": {
   "data": {
    "description": "Hyper parameter tuning",
    "public": false
   },
   "id": "7d1ace18a82178e15ece8fc5252fce88"
  },
  "kernelspec": {
   "display_name": "Python [conda env:p3]",
   "language": "python",
   "name": "conda-env-p3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
